# -*- coding: utf-8 -*-
import os 
import re
#import sgmllib
import socket 
import urllib2

#class URLLister(sgmllib.SGMLParser):
#    def reset(self):                            
#        sgmllib.SGMLParser.reset(self)
#        self.urls = []
#
#    def start_a(self, attrs):                    
#        href = [v for k, v in attrs if k=='href'] 
#        if href:
#            self.urls.extend(href)


def getSD(url, mainDomain ):
    print 'getSD:' + url
    sd = {}
    pstr = ''
    if mainDomain.find("www.") == 0 :
        pstr = mainDomain[4:]
    else :
        pstr = mainDomain

    pstr = r'[\w]+\.' + pstr
    #print pstr
    p=re.compile( pstr, re.I )
    global f
    try:
        f = urllib2.urlopen("http://"+url+"/", timeout=10)

        if f.code == 200:
            content = f.read()
            tempList = p.findall(content)
            if tempList :
                 for item in tempList:
                     tempStr = item.lower()
                     sd[tempStr] = ''
 
            #parser = URLLister()
            #parser.feed(f.read() )
            #for one_url in parser.urls : 
            #    #print one_url
            #    tempList = p.findall(one_url)
            #    if tempList :
            #         for item in tempList:
            #             tempStr = item.lower()
            #             sd[tempStr] = ''
        else :
            sd[url] = 'error'
    except urllib2.HTTPError:
        sd[url] = 'HTTPError at ' + url
    except urllib2.URLError:
        sd[url] = 'URLError at ' + url
    except socket.timeout:
        sd[url] = 'socket.timeout at ' + url
    except socket.error :
        sd[url] = 'socket.error at ' + url
    except :
        sd[url] = 'error at ' + url
    finally :
        f.close()

    #print sd
    return sd

def isSubDomain( sub, main ):
    index = sub.find(main)
    if index > -1 :
        if index == 0 :
            return True
        elif sub[index-1] == '.' :
            return True
        else :
            return False
    else :
        return False

subDomain = {}
#subD = getSD('www.baidu.com')
#for url in subD.keys():
#    subDomain.update( getSD(url) )
#    print subDomain

# 检查是 develop 还是 production ，以确定目录
path = 'main_site_list.txt' 
workpath = '../workspace/'
if os.path.exists(path) :
    workpath = ''


# 读 main_site_list.txt 到 list 中
file_obj = open(workpath + 'main_site_list.txt')
main_site_list = []
for line in file_obj :
    if line.find('\n'):
        line = line[:-1]
    main_site_list.append(line)
file_obj.close()

# 读 sub_domain_list.txt 到 list 中
file_obj = open(workpath + 'sub_domain_list.txt')
sub_domain_list = []
for line in file_obj :
    if line.find('\n'):
        line = line[:-1]
    sub_domain_list.append(line)
file_obj.close()

# 循环处理每个域名，先从list中取出子域名
for site in main_site_list :
    if site.find('\n'):
        site = site[:-1]
    site = site.strip()

    domains = site.split(' ')

    mainDomain = domains[0]
    if mainDomain .find("www.") == 0 :
        mainDomain  = mainDomain [4:]

    prev_SD = {}
    for sub in sub_domain_list :
        if isSubDomain(sub, mainDomain ) :
            prev_SD[sub] = ''

    curr_SD = {}
    curr_SD[mainDomain] = ''
    i = 0
    while i < len(domains) :
        subD = getSD( domains[i], mainDomain  )
        curr_SD.update(subD)
        i += 1

    for sub in prev_SD :
        subD = getSD( sub, mainDomain  )
        curr_SD.update(subD)

    print 
    msg_1 = 'prev_SD:' + str(len(prev_SD)) + '\tcurr_SD:' + str(len(curr_SD)+1) + '\t' + site 
    print msg_1

    # 将子域名写入文件
    subfile = open(workpath + 'curr_sub_domain_list.txt', 'a' )
    workfile = open(workpath + 'subdomain/' + mainDomain  + '.txt', 'w' )
    
    subfile.write( mainDomain + '\n' )

    workfile.write( mainDomain + '\n' )
    workfile.write( msg_1 + '\n' )
    workfile.write('----\n' )
    
    keys = curr_SD.keys()
    keys.sort()
    for key in keys :
        workfile.write( key + ' ' + curr_SD[key] + '\n' )
        subfile.write( key + '\n' )

    subfile.close()
    workfile.close()




#count = 0
#for line in sub_domain_list :
#    count += 1
#    print str(count) + ' ' +line

